package pipeline

import (
	"MyWebcrawler/module"
	"bufio"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
)

func GenFunc(pre string) []module.ProcessItem {
	save := func(item module.Item) (result module.Item, err error) {

		temp := item
		title := temp["title"].(string)
		imgUrl := temp["imgUrl"].(string)
		fileName := path.Base(imgUrl)
		fileDir := pre + title
		_, err = os.Stat(fileDir)
		if err != nil {
			fmt.Errorf("Mkdir")
			os.MkdirAll(fileDir, 0666)
		}
		fileDir = fileDir + "/"
		res, err := http.Get(imgUrl)
		if err != nil {
			fmt.Println("A error occurred!")
			return nil, err
		}
		defer res.Body.Close()

		file, err := os.Create(fileDir + fileName)
		if err != nil {
			panic(err)
		}
		// 获得文件的writer对象
		writer := bufio.NewWriter(file)

		written, _ := io.Copy(writer, res.Body)
		fmt.Printf("Total length: %d\n", written)
		return nil, nil
	}
	return []module.ProcessItem{save}
}
